package com.terren.spider.core.html.processor;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.entity.biz.ArticleSearch;
import com.terren.spider.entity.config.SysConfig;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;

public class HtmlArticleSearchPageProcessor extends BasicHtmlProcessor implements PageProcessor {
	
	protected List<ArticleSearch> articleList = new ArrayList<>();//存放搜索页的标题，链接，发布时间等信息

	public HtmlArticleSearchPageProcessor(Configuration configuration) throws Exception {
		super(configuration);
	}

	@Override
	public void process(Page page) {
		logMessage(page);
		/**
		 * 目标页（即列表页）
		 */
//		if (!isExcludeTargetUrl(page, entry,configuration.getContentUseProxy())) {//不是排除的目标页，是目标页。
			try {
				/***
				 * 加入存储队列
				 */
				Page collectItem = page;
				page.putField(SysConfig.COLLECT_ITEM_KEY, collectItem);// 把page对象放入pipeline 等待分析
				page.putField(SysConfig.COLLECT_TOTAL_TARGET_URL, getTargetUrlCount());// 把 所有条目放入pipeline 等待分析
				
				for (ArticleSearch article : articleList) {
					if (getCurrentPageUrl(collectItem).equals(article.getUrl())) {//url 匹配
						page.putField(SysConfig.COLLECT_ITEM_TITLE_KEY, article.getTitle());// 把 title  放入pipeline 等待分析
						page.putField(SysConfig.COLLECT_ITEM_PUBTIME_KEY, article.getPubTimeStr());// 把 pubtime  放入pipeline 等待分析
						page.putField(SysConfig.COLLECT_ITEM_SOURCE_KEY, article.getNewsSource());// 把来源  放入pipeline 等待分析
						page.putField(SysConfig.COLLECT_ITEM_AUTHOR_KEY, article.getAuthor());//作者
						page.putField(SysConfig.COLLECT_ITEM_AUTHOR_URL_KEY, article.getAuthorUrl());//作者 链接
						/**
						 * 百度搜索得到的链接不是真是页面地址，需要跳转，所以从page对象中取出
						 */
						String realUrl = page.getRequest().getExtra(SysConfig.LAST_REDIRECT_URL) != null ? (String) page.getRequest().getExtra(SysConfig.LAST_REDIRECT_URL)
								: null;
						String targetUrl = page.getRequest().getExtra("url") != null ? (String) page.getRequest().getExtra("url")
								: null;
						String url = realUrl!=null? realUrl : targetUrl != null ? targetUrl: page.getUrl().toString();
						
						
						page.putField(SysConfig.COLLECT_ITEM_LAST_REDIRECT_URL, url);// 把来源  放入pipeline 等待分析
					}
				}
				
				Set<String> paginationUrl = new HashSet<>();
				paginationUrl.addAll(this.findTargetPaginationUrl(page,  configuration));// 分页
				this.addTargetRequests(page,paginationUrl);
			} catch (Exception e) {
				log.error("process parse error", e);
			}
		//}

	}

	public List<ArticleSearch> getArticleList() {
		return articleList;
	}

	public void setArticleList(List<ArticleSearch> articleList) {
		this.articleList = articleList;
	}
}
